{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "({0: 'O',\n  1: 'B-地块编码',\n  2: 'I-地块编码',\n  3: 'B-地块位置',\n  4: 'I-地块位置',\n  5: 'B-出让面积',\n  6: 'I-出让面积',\n  7: 'B-土地用途',\n  8: 'I-土地用途',\n  9: 'B-容积率',\n  10: 'I-容积率',\n  11: 'B-起始价',\n  12: 'I-起始价',\n  13: 'B-成交价',\n  14: 'I-成交价',\n  15: 'B-溢价率',\n  16: 'I-溢价率',\n  17: 'B-成交时间',\n  18: 'I-成交时间',\n  19: 'B-受让人',\n  20: 'I-受让人',\n  21: 'B-城市',\n  22: 'I-城市',\n  23: 'B-触发词',\n  24: 'I-触发词'},\n {'O': 0,\n  'B-地块编码': 1,\n  'I-地块编码': 2,\n  'B-地块位置': 3,\n  'I-地块位置': 4,\n  'B-出让面积': 5,\n  'I-出让面积': 6,\n  'B-土地用途': 7,\n  'I-土地用途': 8,\n  'B-容积率': 9,\n  'I-容积率': 10,\n  'B-起始价': 11,\n  'I-起始价': 12,\n  'B-成交价': 13,\n  'I-成交价': 14,\n  'B-溢价率': 15,\n  'I-溢价率': 16,\n  'B-成交时间': 17,\n  'I-成交时间': 18,\n  'B-受让人': 19,\n  'I-受让人': 20,\n  'B-城市': 21,\n  'I-城市': 22,\n  'B-触发词': 23,\n  'I-触发词': 24})"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trigger_label = '触发词'\n",
    "schemas = ['O', 'B-地块编码','I-地块编码', 'B-地块位置', 'I-地块位置', 'B-出让面积', 'I-出让面积', 'B-土地用途', 'I-土地用途', 'B-容积率', 'I-容积率',\n",
    "           'B-起始价', 'I-起始价', 'B-成交价', 'I-成交价', 'B-溢价率', 'I-溢价率', 'B-成交时间', 'I-成交时间', 'B-受让人', 'I-受让人',\n",
    "           'B-城市', 'I-城市', 'B-触发词', 'I-触发词']\n",
    "id2tag = {idx: tag for idx, tag in enumerate(schemas)}\n",
    "tag2id = {tag: idx for idx, tag in enumerate(schemas)}\n",
    "\n",
    "id2tag,tag2id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [],
   "source": [
    "# hide_output\n",
    "from transformers import AutoTokenizer\n",
    "#加载\n",
    "bert_model_name = \"bert-base-chinese\"\n",
    "bert_tokenizer = AutoTokenizer.from_pretrained(bert_model_name)\n",
    "def tokenize_and_align_labels(examples):\n",
    "    #输入字的list，输出用tokenizer的词典中序号表示的字，并对特殊字符和子词进行特殊处理。\n",
    "    tokenized_inputs = bert_tokenizer(examples[\"tokens\"], truncation=True,\n",
    "                                      is_split_into_words=True)\n",
    "    #定义空的标签表示\n",
    "    labels = []\n",
    "    #迭代输入的ner_tags标签\n",
    "    for idx, label in enumerate(examples[\"ner_tags\"]):\n",
    "        #tokenized_input包含word_ids函数，实现对子词与整词的识别。\n",
    "        #这里我们可以看到word_ids将每个子单词映射到单词序列中对应的索引\n",
    "        word_ids = tokenized_inputs.word_ids(batch_index=idx)\n",
    "        previous_word_idx = None\n",
    "        label_ids = []\n",
    "        for word_idx in word_ids:\n",
    "            # 将特殊符号的标签设置为-100，以便在计算损失函数时自动忽略\n",
    "            if word_idx is None:\n",
    "                label_ids.append(-100)\n",
    "            # 把标签设置到每个词的第一个token上\n",
    "            elif word_idx != previous_word_idx:\n",
    "                label_ids.append(label[word_idx])\n",
    "            # 对于每个词的其他token也设置为当前标签\n",
    "            else:\n",
    "                label_ids.append(label[word_idx])\n",
    "            previous_word_idx = word_idx\n",
    "\n",
    "        labels.append(label_ids)\n",
    "    #把处理后的labels数组，设置为tokenized_inputs[\"labels\"]\n",
    "    tokenized_inputs[\"labels\"] = labels\n",
    "    return tokenized_inputs"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default-9f377d3292216168\n",
      "Reusing dataset csv (C:\\Users\\xiaofei\\.cache\\huggingface\\datasets\\csv\\default-9f377d3292216168\\0.0.0\\bf68a4c4aefa545d0712b2fcbb1b327f905bbe2f6425fbc5e8c25234acb9e14a)\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/2 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "073bd2914d0f4c1f9c95e5197d7f1cc5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "'['"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取自定义数据集\n",
    "from datasets import load_dataset\n",
    "land = load_dataset(\"csv\", data_files = {\n",
    "    'train': ['train.csv'],\n",
    "    'validation' : ['valid.csv']\n",
    "})\n",
    "land[\"train\"][\"tokens\"][0][0]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "\u001B[1;32m~\\AppData\\Local\\Temp/ipykernel_12160/3465047737.py\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[1;32m----> 1\u001B[1;33m \u001B[0mtokenized_input\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mtokenize_and_align_labels\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mland\u001B[0m\u001B[1;33m[\u001B[0m\u001B[1;34m\"train\"\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m      2\u001B[0m \u001B[0mtokenized_input\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\AppData\\Local\\Temp/ipykernel_12160/960308239.py\u001B[0m in \u001B[0;36mtokenize_and_align_labels\u001B[1;34m(examples)\u001B[0m\n\u001B[0;32m     17\u001B[0m         \u001B[0mprevious_word_idx\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;32mNone\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     18\u001B[0m         \u001B[0mlabel_ids\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;33m[\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 19\u001B[1;33m         \u001B[1;32mfor\u001B[0m \u001B[0mword_idx\u001B[0m \u001B[1;32min\u001B[0m \u001B[0mword_ids\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     20\u001B[0m             \u001B[1;31m# 将特殊符号的标签设置为-100，以便在计算损失函数时自动忽略\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     21\u001B[0m             \u001B[1;32mif\u001B[0m \u001B[0mword_idx\u001B[0m \u001B[1;32mis\u001B[0m \u001B[1;32mNone\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\AppData\\Local\\Temp/ipykernel_12160/960308239.py\u001B[0m in \u001B[0;36mtokenize_and_align_labels\u001B[1;34m(examples)\u001B[0m\n\u001B[0;32m     17\u001B[0m         \u001B[0mprevious_word_idx\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;32mNone\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     18\u001B[0m         \u001B[0mlabel_ids\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;33m[\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 19\u001B[1;33m         \u001B[1;32mfor\u001B[0m \u001B[0mword_idx\u001B[0m \u001B[1;32min\u001B[0m \u001B[0mword_ids\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     20\u001B[0m             \u001B[1;31m# 将特殊符号的标签设置为-100，以便在计算损失函数时自动忽略\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     21\u001B[0m             \u001B[1;32mif\u001B[0m \u001B[0mword_idx\u001B[0m \u001B[1;32mis\u001B[0m \u001B[1;32mNone\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m_pydevd_bundle\\pydevd_cython_win32_39_64.pyx\u001B[0m in \u001B[0;36m_pydevd_bundle.pydevd_cython_win32_39_64.SafeCallWrapper.__call__\u001B[1;34m()\u001B[0m\n",
      "\u001B[1;32m_pydevd_bundle\\pydevd_cython_win32_39_64.pyx\u001B[0m in \u001B[0;36m_pydevd_bundle.pydevd_cython_win32_39_64.PyDBFrame.trace_dispatch\u001B[1;34m()\u001B[0m\n",
      "\u001B[1;32m_pydevd_bundle\\pydevd_cython_win32_39_64.pyx\u001B[0m in \u001B[0;36m_pydevd_bundle.pydevd_cython_win32_39_64.PyDBFrame.trace_dispatch\u001B[1;34m()\u001B[0m\n",
      "\u001B[1;32m_pydevd_bundle\\pydevd_cython_win32_39_64.pyx\u001B[0m in \u001B[0;36m_pydevd_bundle.pydevd_cython_win32_39_64.PyDBFrame.trace_dispatch\u001B[1;34m()\u001B[0m\n",
      "\u001B[1;32mD:\\Program Files\\JetBrains\\PyCharm 2022.2\\plugins\\python\\helpers-pro\\jupyter_debug\\pydev_jupyter_plugin.py\u001B[0m in \u001B[0;36mstop\u001B[1;34m(plugin, pydb, frame, event, args, stop_info, arg, step_cmd)\u001B[0m\n\u001B[0;32m    167\u001B[0m         \u001B[0mframe\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0msuspend_jupyter\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mmain_debugger\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mthread\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mframe\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mstep_cmd\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    168\u001B[0m         \u001B[1;32mif\u001B[0m \u001B[0mframe\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 169\u001B[1;33m             \u001B[0mmain_debugger\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mdo_wait_suspend\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mthread\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mframe\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mevent\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0marg\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    170\u001B[0m             \u001B[1;32mreturn\u001B[0m \u001B[1;32mTrue\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    171\u001B[0m     \u001B[1;32mreturn\u001B[0m \u001B[1;32mFalse\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mD:\\Program Files\\JetBrains\\PyCharm 2022.2\\plugins\\python\\helpers\\pydev\\pydevd.py\u001B[0m in \u001B[0;36mdo_wait_suspend\u001B[1;34m(self, thread, frame, event, arg, send_suspend_message, is_unhandled_exception)\u001B[0m\n\u001B[0;32m   1158\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1159\u001B[0m         \u001B[1;32mwith\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_threads_suspended_single_notification\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mnotify_thread_suspended\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mthread_id\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mstop_reason\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 1160\u001B[1;33m             \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_do_wait_suspend\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mthread\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mframe\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mevent\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0marg\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0msuspend_type\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mfrom_this_thread\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   1161\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1162\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0m_do_wait_suspend\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mthread\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mframe\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mevent\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0marg\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0msuspend_type\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mfrom_this_thread\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mD:\\Program Files\\JetBrains\\PyCharm 2022.2\\plugins\\python\\helpers\\pydev\\pydevd.py\u001B[0m in \u001B[0;36m_do_wait_suspend\u001B[1;34m(self, thread, frame, event, arg, suspend_type, from_this_thread)\u001B[0m\n\u001B[0;32m   1173\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1174\u001B[0m                 \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mprocess_internal_commands\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 1175\u001B[1;33m                 \u001B[0mtime\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0msleep\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;36m0.01\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   1176\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1177\u001B[0m         \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mcancel_async_evaluation\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mget_current_thread_id\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mthread\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mid\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mframe\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "source": [
    "tokenized_input = tokenize_and_align_labels(land[\"train\"])\n",
    "tokenized_input"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "example = land[\"train\"][0]\n",
    "print(example)\n",
    "tokenized_input = bert_tokenizer(example[\"data\"])\n",
    "print(tokenized_input)\n",
    "tokens = bert_tokenizer.convert_ids_to_tokens(tokenized_input[\"input_ids\"])\n",
    "word_ids = tokenized_input.word_ids()\n",
    "pd.DataFrame([tokens, word_ids], index=[\"Tokens\", \"Word IDs\"])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}